import numpy as np
import cv2
import matplotlib.pyplot as plt
from moviepy.editor import VideoFileClip
import glob
#show matplot images inline
%matplotlib inline

Get images

def show_image(image,title="image",cmap_type="gray",axis='off'):
    plt.imshow(image,cmap_type)
    plt.title(title)
    plt.axis(axis)

#LOADING ALL IMAGES FROM TEST_IMAGES FOLDER
images = glob.glob("test_images/*.jpg")
images = [plt.imread(image) for image in images] 

# show_image(images[0],"FIRST_IMAGE")
#Displays 8 images in 4 rows and 2 columns
def display_images(images, cmap='gray'):
    plt.figure(figsize=(40,40))
    for i, image in enumerate(images):
        plt.subplot(5,2,i+1)
        plt.imshow(image, cmap)
        plt.autoscale(tight=True)
    plt.show()
display_images(images,None)

Helper functions

# Histogram Equalizer
def hist_equalizer(img):
    equ = cv2.equalizeHist(img)
    return equ
# gets histogram below y/2
def get_hist(img):
    hist = np.sum(img[img.shape[0]//2:,:], axis=0)
    return hist

# averaging filter
def gaussian_blur(img,kernel_size=5):
    return cv2.GaussianBlur(img,(kernel_size,kernel_size),0)

# draws region of interest
def draw_roi(img,isClosed=True,color=(255,0,0),thickness=5):
    x,y = (img.shape[1],img.shape[0])
    pts = np.array([[0.17*x,0.95*y], [0.43*x,int(0.65*y)],
                [0.58*x,0.65*y], [1*x,0.95*y]],
               np.int32)
    pts = pts.reshape((-1, 1, 2))
    img = cv2.polylines(img, [pts], 
                      isClosed, color, 
                      thickness)
    return img
# gets birdeye view
def perspective_transform(img,dst_size=(1280,720),inv=0):
    img_size=np.float32([(img.shape[1],img.shape[0])])
    #Order is top left, top right, bottom left, bottom right
    src=np.float32([(0.43,0.65),(0.58,0.65),(0.15,0.95),(0.95,0.95)])
    dst=np.float32([(0,0), (1, 0), (0,1), (1,1)])
    srcPoints = src*img_size
    dstPoints = dst*np.float32(dst_size)
    if(inv):
        M = cv2.getPerspectiveTransform(dstPoints,srcPoints) #inverse
    else:
        M = cv2.getPerspectiveTransform(srcPoints,dstPoints) #Returns a matrix that transforms an Image
    warped_image = cv2.warpPerspective(img,M,dst_size)
    return warped_image 

# enhances bright objects of interest in a dark background
def top_hat_filter(img):
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(30,3))
    tophat = cv2.morphologyEx(img, cv2.MORPH_TOPHAT, kernel)
    #convert to binary
    mask = np.zeros_like(tophat)
    mask[((tophat >= 10)&(tophat<=150))] = 1
    return mask

#main pipeline
def lane_filter(img):
    img = np.copy(img)
    # Convert to HLS and LAB color spaces
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    hls = gaussian_blur(hls,5)
    lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)
    lab = gaussian_blur(lab,5)
    
    # Separate the l channel from lab color space
    l_channel = lab[:,:,0]
    # Separate the s channel from HLS color space
    s_channel = hls[:,:,2]
    
    #l_channel works relatively good under bridge and detects white lines
    tophat = top_hat_filter(l_channel)
    
    #Works well  to differentiate colors at sun/dirt environment
    s_thresh=(100, 255)
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
    
    #combined_binary combines tophat and s_binary by oring
    combined_binary = np.zeros_like(tophat)
    # to get details from both channels
    combined_binary[(s_binary == 1) | (tophat == 1)] = 1
    return combined_binary

Applying filter and prespective transform

output = list(map(lane_filter, images))
display_images(output)

def draw_roi(img,isClosed=True,color=(255,0,0),thickness=5):
    x,y = (img.shape[1],img.shape[0])
    pts = np.array([[0.15*x,0.95*y], [0.43*x,int(0.65*y)],
                [0.58*x,0.65*y], [1*x,0.95*y]],
               np.int32)
    pts = pts.reshape((-1, 1, 2))
    img = cv2.polylines(img, [pts], 
                      isClosed, color, 
                      thickness)
    return img

# test = np.copy(images[0])
# draw_ro = draw_roi(test)
# show_image(draw_ro)

warped_prespective = list(map(perspective_transform, output))
display_images(warped_prespective)

Sliding Window

#Globally defined to store the parameters of past images
left_a, left_b, left_c = [],[],[]
right_a, right_b, right_c = [],[],[]

def sliding_window(img, 
                   nwindows=9, #Number of windows 
                   margin=100, #half window width 100?
                   minpix = 1 #minimum number of pixels to recenter the window 50? 
                  ):
    # left and right curve parameters of all frames
    global left_a, left_b, left_c,right_a, right_b, right_c  # global keyword uses the global variables
    
    # current left and right curve parameters
    left_fit_= np.empty(3) #parameters of 2nd order polynomial
    right_fit_ = np.empty(3) 
    out_img = np.dstack((img, img, img))*255 #Converts grayscale to 3 dimensional channel normal RGB
    
    #USING HISTOGRAM METHOD
    histogram = get_hist(img) 
    # find peaks of left and right halves
    midpoint = int(histogram.shape[0]/2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint
    
    # Set height of windows
    window_height = np.int32(img.shape[0]/nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    # Indices in image is the coordinates the row is the height is the y
    # the column is the width is the x
    nonzero = img.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated for each window
    leftx_current = leftx_base
    rightx_current = rightx_base
    
    
    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = img.shape[0] - (window+1)*window_height
        win_y_high = img.shape[0] - window*window_height
        win_xleft_low = leftx_current - margin
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin
        
        # Draw the windows on the visualization image
        cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),
        (0,255,0), 3) 
        cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),
        (0,255,0), 3) 
        
        #Get non-zero pixels within each window by getting the indices of nonzerox
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0] 
        #[0] get indices of nonzerox only but you can access nonzeroy with it too
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]
        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        ''' 
        minpix is used to recenter the window and depends on how good the filter is
        if minpix is low it might be affected by the noise(?)
        therefore we keep minpix at 50 pixels for now to only recenter in the direction of any solid line
        '''
        # If you found > minpix pixels, recenter next window on their mean position
        if len(good_left_inds) > minpix:
            leftx_current = np.int32(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:        
            rightx_current = np.int32(np.mean(nonzerox[good_right_inds]))
        

    # Concatenate the arrays of indices
    #All pixels within all 9 windows
    left_lane_inds = np.concatenate(left_lane_inds) 
    right_lane_inds = np.concatenate(right_lane_inds)

    # Extract left and right line pixel positions
    #All x and y coordinates for pixels residing inside the 9 windows
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds] 

    # Fit a second order polynomial to each
    #Returns three parameters 
    left_fit = np.polyfit(lefty, leftx,
                          2 #Order of the equation
                         )
    right_fit = np.polyfit(righty, rightx, 2)
    
    #ay^2+by+c
    #store these three parameters
    left_a.append(left_fit[0])
    left_b.append(left_fit[1])
    left_c.append(left_fit[2])
    
    right_a.append(right_fit[0])
    right_b.append(right_fit[1])
    right_c.append(right_fit[2])
    
    #find the mean for the last 10 frames
    left_fit_[0] = np.mean(left_a[-10:])
    left_fit_[1] = np.mean(left_b[-10:])
    left_fit_[2] = np.mean(left_c[-10:])
    
    right_fit_[0] = np.mean(right_a[-10:])
    right_fit_[1] = np.mean(right_b[-10:])
    right_fit_[2] = np.mean(right_c[-10:])
    
    # Generate x and y values for plotting the two curves
    ploty = np.linspace(0, img.shape[0]-1, img.shape[0] ) #the points that will be on the y-axis
    left_fitx = left_fit_[0]*ploty**2 + left_fit_[1]*ploty + left_fit_[2] 
    right_fitx = right_fit_[0]*ploty**2 + right_fit_[1]*ploty + right_fit_[2]
    
    return out_img, (left_fitx, right_fitx), (left_fit_, right_fit_), ploty

#Overlay on frames
def draw_lanes(img, left_fit, right_fit,ploty):
#     ploty = np.linspace(0, img.shape[0]-1, img.shape[0])
    color_img = np.zeros_like(img)
    #Horizontal stack #dstack = depth stack #vstack = vertical stack
    left = np.array([np.transpose(np.vstack([left_fit, ploty]))])
    right = np.array([np.flipud(np.transpose(np.vstack([right_fit, ploty])))]) #flip up to down
    points = np.hstack((left, right)) 
    #Draw two curved lines
    cv2.polylines(color_img, np.int_(left) , False, (255,0,0),50) #Red curved line
    cv2.polylines(color_img, np.int_(right), False, (0,0,255),50) #Blue curved line
    #Draw a polygon of the curve shape
    cv2.fillPoly(color_img, np.int_(points), (0,255,0)) #Green curve
    inv_perspective = perspective_transform(color_img,inv=1)
    inv_perspective = cv2.addWeighted(img, 1, inv_perspective, 0.7, 0)
    return inv_perspective,color_img

def get_curve(img, leftx, rightx):
    # get image y-axis
    # linspace takes start, stop, number of steps
    # so here we will start from 0, reach to image last y point
    # and takes steps equal to all image y pixels
    ploty = np.linspace(0, img.shape[0]-1, img.shape[0])
    y_eval = np.max(ploty)
    
    # these values are related to camera
    ym_per_pix = 30.5/720 # meters per pixel in y dimension
    xm_per_pix = 3.7/730 # meters per pixel in x dimension

    # Fit new polynomials to x,y in world space
    left_fit_cr = np.polyfit(ploty*ym_per_pix, leftx*xm_per_pix, 2)
    right_fit_cr = np.polyfit(ploty*ym_per_pix, rightx*xm_per_pix, 2)
    
    # Calculate the new radii of curvature
    left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
    right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])

    # center of x-axis
    car_pos = img.shape[1]/2
    
    l_fit_x_int = left_fit_cr[0]*img.shape[0]**2 + left_fit_cr[1]*img.shape[0] + left_fit_cr[2]
    r_fit_x_int = right_fit_cr[0]*img.shape[0]**2 + right_fit_cr[1]*img.shape[0] + right_fit_cr[2]
    lane_center_position = (r_fit_x_int + l_fit_x_int) /2
    center = (car_pos - lane_center_position) * xm_per_pix / 10

    return (left_curverad, right_curverad, center)

# def display_sliding_window(images):
#         plt.figure(figsize=(40,40))
#         for i, image in enumerate(images):
#             out_img, x_points, _, ploty = sliding_window(image)
#             img_,birdview_curve = draw_lanes(image, x_points[0], x_points[1],ploty)
#             plt.subplot(9,2,i+1)
#             plt.imshow(out_img)
#             plt.plot(x_points[0], ploty, color='yellow', linewidth=20)
#             plt.plot(x_points[1], ploty, color='yellow', linewidth=20)
#             plt.imshow(img_)
#             plt.autoscale(tight=True)
#         plt.show()
def cell_sliding_window(warped_image,image):
    warped_image[int(warped_image.shape[0]//2):warped_image.shape[0],400:900] = 0
    out_img, x_points, _, ploty = sliding_window(warped_image)
    img_,birdview_curve = draw_lanes(image, x_points[0], x_points[1],ploty)
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(100, 20))
    ax1.imshow(out_img)
    ax1.plot(x_points[0], ploty, color='red', linewidth=30)
    ax1.plot(x_points[1], ploty, color='blue', linewidth=30)
    ax1.set_title('Curves on sliding window', fontsize=100)
    ax2.imshow(img_)
    ax2.set_title('Original image with overlay', fontsize=100)
    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

for i,image in enumerate(images):
    cell_sliding_window(warped_prespective[i],image)
    

def debugging_video_frame_by_frame(path):
    ##Live Video Capture Option for debugging mode
    #Note: previous image function not implemented yet
    cap = cv2.VideoCapture(path)
    _,frame = cap.read()
    while(cap.isOpened()):
        action = cv2.waitKey(1) & 0xFF
        if(action == ord('q')):
            break
        #press l to get next image
        if((action == ord('l'))):
            _,frame = cap.read()
#        frames.put(frame)
#        convert frame to RGB first
#         frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        combo_image = final_output(frame)
#        convert combo_image to BGR before display
 
# the image outputs image size more than 1920x1080 and imshow doesn't allow resizing(?)
        img_size = (frame.shape[1],frame.shape[0])
        combo_image = cv2.resize(combo_image,img_size)
#         combo_image = cv2.cvtColor(combo_image, cv2.COLOR_RGB2BGR)
        #######################
        cv2.imshow("result",combo_image)
        
    cap.release()
    cv2.destroyAllWindows()
def final_output(img):
    # define font style
    font = cv2.FONT_HERSHEY_SIMPLEX
    fontColor = (52, 119, 235) #RGB
    fontSize=2
    textPosition = (50,50)
    
    img_roi = np.copy(img)
    img_roi = draw_roi(img_roi) #Region of interest 1
    
    img_pip = lane_filter(img) #Returns binary image s_condition + l_channel tophat
    img_pip2 = np.dstack((img_pip,img_pip,img_pip))*255
    cv2.putText(img_pip2, 'Filtered Image', textPosition, font, fontSize, fontColor, 2)
    
   # Birdview
    img_pt = perspective_transform(img_pip, dst_size=(1280,720),inv=0) #birdview
    img_pt[int(img_pt.shape[0]//2):img_pt.shape[0],400:900] = 0
#     img_pt[int(img_pt.shape[0]/2):img_pt.shape[0],250:1000] = 0
    img_pt2 = np.dstack((img_pt,img_pt,img_pt))*255
    cv2.putText(img_pt2, 'Bird eye view', textPosition, font, fontSize, fontColor, 2)

    img_sw, curves, _, ploty = sliding_window(img_pt) #Sliding window image + curve points and parameters 
    cv2.putText(img_sw, 'Sliding window result', textPosition, font, fontSize, fontColor, 2)


    # get curves
    curverad =get_curve(img, curves[0], curves[1])
    lane_curve = np.mean([curverad[0], curverad[1]])

    img_final,birdview_curve = draw_lanes(img, curves[0], curves[1],ploty) #draws the overlay
    cv2.putText(birdview_curve, 'Polynomial fit', textPosition, font, fontSize, fontColor, 2)
    #Debugging mode?
    # resize images
    #Bottom
    img_bot = np.hstack((img_sw,birdview_curve)) #Same width but half height
    img_bot = cv2.resize(img_bot, (img_final.shape[1],int(img_final.shape[0]/2)))
    img = np.vstack((img_final,img_bot))

    #Side
    img_stack = np.vstack((img_pip2,img_pt2))
    
    # img_stack = np.dstack((img_stack,img_stack,img_stack))*255
    
    img_side = np.vstack((img_roi,img_stack))
    img_side = cv2.resize(img_side, (int(img.shape[1]/2),img.shape[0]))
    img_final = np.hstack((img,img_side))

    cv2.putText(img_final, 'Lane Curvature: {:.0f} m'.format(lane_curve), (50, 50), font, 1, fontColor, 2)
    cv2.putText(img_final, 'Vehicle offset: {:.4f} m'.format(curverad[2]), (50, 80), font, 1, fontColor, 2)
    return img_final

Test final_output & live debugging

# When the video starts:
# Press "L" to get next frame
# Press "Q" to close video
debugging_video_frame_by_frame('project_video.mp4')

myclip = VideoFileClip('project_video.mp4')#.subclip(40,43)
output_vid = 'project_video1.mp4'
clip = myclip.fl_image(final_output)
clip.write_videofile(output_vid, fps =25,audio=False)

myclip = VideoFileClip('challenge_video.mp4')#.subclip(40,43)
output_vid = 'challenge_video1.mp4'
clip = myclip.fl_image(final_output)
clip.write_videofile(output_vid, fps =25,audio=False)